﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using VulnDBExplorer.Framework.Impl;
using System.IO;
using System.Threading;
using Luna;
using Winista.Text.HtmlParser;
using Winista.Text.HtmlParser.Tags;
using Winista.Text.HtmlParser.Lex;
using System.Text.RegularExpressions;
using System.Diagnostics;
using System.Net;

namespace VulnDB.Action.DataCollectors.Actions {

	class PageJob {
		public int Page { get; set; }
		public string Data { get; set; }
	}

	class TestSuiteCollectorAction : MultiThreadDataCollector<PageJob> {
		private string baseURL;
		private object pageSync = new object();
		private CookieContainer container;

		public TestSuiteCollectorAction() {
			this.Name = "FirefoxTestSuiteCollectorAction";
			this.Text = "Collect Firefox TestSuite";

			this.ExecutionMode = ActionExecutionMode.SeparatedThread;
			//TestSuitURL = "https://litmus.mozilla.org/advanced_search.cgi?product=1&limit={0}&automated=all&withbugs=all";
			TestSuitURL = "https://litmus.mozilla.org/search_results.cgi?product=1&platform=&locale=&limit={0}";
			Output = "output/ff-testsuite.{0}.cvs";
			ItemPerPage = 2000;
			NumberOfWorker = 4;
			StartingPage = 1;
			PageCount = int.MaxValue;
			MaxQueueCount = 20;
		}

		public string TestSuitURL { get; set; }

		public string QueryArguments { get; set; }

		public string Output { get; set; }

		public int ItemPerPage { get; set; }

		public int StartingPage { get; set; }

		public int PageCount { get; set; }

		public int ProcessedPages { get; set; }

		public int TestResultCount { get; set; }

		protected override void setupExecution(Luna.Framework.ActionExecutionArgs args) {
			Console.log("Collecting Firefox TestSuite...");
			ProgressMonitor = Application.createProgressMonitor();
			ProgressMonitor.start("Collecting test suite...", int.MaxValue);
			baseURL = TestSuitURL.format(ItemPerPage) + QueryArguments;
			container = new CookieContainer();
		}

		protected override void finalizeExecution() {
			if (ProgressMonitor != null)
				ProgressMonitor.stop();
			if (Console != null) {
				Console.writeLine("Items per page : {0}", ItemPerPage);
				Console.writeLine("Processed      : {0} / {1} pages", ProcessedPages, PageCount != int.MaxValue ? Math.Max(ProcessedPages, PageCount): ProcessedPages);
				Console.writeLine("Test Result collected: {0:#,##0}", TestResultCount);
			}
		}

		protected override object onJobExecute(PageJob job) {
			List<TestSuiteEntry> result = new List<TestSuiteEntry>();
			string data = job.Data;
			int page = job.Page;

			int idx = data.IndexOf("<table class=\"test-results\">");
			if (idx > 0) {
				int eidx = data.IndexOf("</table>", idx);
				data = data.Substring(idx, eidx + "</table>".Length - idx + 1);
				Stopwatch t = new Stopwatch();
				Console.writeLine("Processing page {0}...", page);
				t.Start();
				parseReturnHtml(page, data, result);
				t.Stop();
				Console.writeLine("Page {0} took {1} to processed.", page, t.Elapsed.ToString());
			}

			return result;
		}

		protected override void onJobCompleted(PageJob job, object result) {
			List<TestSuiteEntry> list = (result as List<TestSuiteEntry>);
			lock (this) {
				ProcessedPages++;
				TestResultCount += list.Count;
			}
			using (var f = new StreamWriter(Output.format(job.Page)))
				foreach (TestSuiteEntry entry in list)
					f.WriteLine("{0},{1},{2},{3},{4}", entry.ID, entry.Result, entry.SubmissionDate, entry.TestCaseID, entry.Branch);

			ProgressMonitor.update(ProcessedPages, PageCount);
			ProgressMonitor.Description = "{0} entries collected.".format(TestResultCount);
		}

		protected object onJobExecute(int page) {
			string url = baseURL + (page > 1 ? "&page=" + page.ToString() : "");

			string data = syncLoadPage(url, page);

			if (true || PageCount == int.MaxValue || PageCount == 0) {
				//Matched 245670 results
				Regex reg = new Regex("Matched (\\d+) result", RegexOptions.Multiline);
				Match m = reg.Match(data);
				if (m.Success) {
					int itemCount = m.Groups[1].Value.toInt();
					PageCount = itemCount / ItemPerPage + (itemCount % ItemPerPage > 0 ? 1 : 0);
				}
			}

			List<TestSuiteEntry> result = new List<TestSuiteEntry>();

			int idx = data.IndexOf("<table class=\"test-results\">");
			if (idx > 0) {
				int eidx = data.IndexOf("</table>", idx);
				data = data.Substring(idx, eidx + "</table>".Length - idx + 1);
				Stopwatch t = new Stopwatch();
				Console.writeLine("Processing page {0}...", page);
				t.Start();
				parseReturnHtml(page, data, result);
				t.Stop();
				Console.writeLine("Page {0} took {1} to processed.", page, t.Elapsed.ToString());
			}

			return result;
		}

		protected void onJobCompleted(int job, object result) {
			List<TestSuiteEntry> list = (result as List<TestSuiteEntry>);
			lock (this) {
				ProcessedPages++;
				TestResultCount += list.Count;
			}
			using (var f = new StreamWriter(Output.format(job)))
				foreach(TestSuiteEntry entry in list)
					f.WriteLine("{0},{1},{2},{3},{4}",entry.ID, entry.Result, entry.SubmissionDate, entry.TestCaseID, entry.Branch);

			ProgressMonitor.update(ProcessedPages, PageCount);
			ProgressMonitor.Description = "{0} entries collected.".format(TestResultCount);
		}

		protected IEnumerable<int> enumJobs1() {
			for (int i = StartingPage; i <= PageCount; i++) 
				yield return i;
			
		}

		protected override IEnumerable<PageJob> enumJobs() {
			for (int i = StartingPage; i <= PageCount; i++) {
				string data = syncLoadPage(baseURL, i);
				Regex reg = new Regex("Matched (\\d+) result", RegexOptions.Multiline);
				Match m = reg.Match(data);
				if (m.Success) {
					int itemCount = m.Groups[1].Value.toInt();
					PageCount = itemCount / ItemPerPage + (itemCount % ItemPerPage > 0 ? 1 : 0);
				}
				yield return new PageJob() { Data = data, Page = i };
			}
		}

		private string syncLoadPage(string url, int page) {
			string data = string.Empty;
			string cacheFile = "caches/testsuite-{0}-{1}.cache-htm".format(ItemPerPage, page);
			url = url + (page > 1 ? "&page=" + page.ToString() : "");
			Stopwatch t = new Stopwatch();
			lock (container) {
				Console.writeLine("Loading page {0}...", page);
				t.Start();
				if (cacheFile.fileExists())
					data = File.ReadAllText(cacheFile);
				else {
					HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
					request.Proxy = WebRequest.DefaultWebProxy;
					request.Timeout = 10 * 60 * 1000;
					request.CookieContainer = container;
					using (var res = request.GetResponse())
					using (var ss = res.GetResponseStream())
					using (var s = new StreamReader(ss))
						data = s.ReadToEnd();
					File.WriteAllText(cacheFile, data);
				}
				//return result;
				//data = FileExtension.readWebFile(url, cacheFile);
				t.Stop();
				Console.writeLine("Page {0} took {1} to load.", page, t.Elapsed.ToString());
			}
			return data;
		}

		private void parseReturnHtml(int page, string data, List<TestSuiteEntry> result) {
			HtmlTable table = new HtmlTable(data);

			for (int i = 2; i < table.Rows.Count - 1; i++) {
				HtmlTableRow tr = table.Rows[i];
				try {
					TestSuiteEntry entry = new TestSuiteEntry();
					entry.Result = tr.getAttributte("class").convertTo<TestSuiteEntryResult>(TestSuiteEntryResult.NotSpecify);
					entry.SubmissionDate = tr.Columns[0].ChildrenHtml.convertTo<DateTime>(DateTime.MinValue);
					entry.ID = extractNumber(tr.Columns[1].ChildrenHtml);
					entry.TestCaseID = extractNumber(tr.Columns[2].ChildrenHtml);
					entry.Branch = tr.Columns[5].ChildrenHtml;
					result.Add(entry);
				}
				catch (Exception) {
					break;
				}
			}
		}

		private int extractNumber(string text) {
			Match m = Regex.Match(text, "\\d+");
			if (m.Success)
				return m.Value.toInt();
			return 0;
		}

		private void parseReturnHtmlSlow(int page, string data, List<TestSuiteEntry> result) {
			Parser parser = new Parser(new Lexer(data));
			INode[] nodes = parser.ExtractAllNodesThatAre(typeof(TableTag));
			TableTag tbl = (TableTag)nodes[0];
			
			for (int i = 2; i < tbl.RowCount - 1; i++) {
				TableRow tr = tbl.Rows[i];
				try {
					TestSuiteEntry entry = new TestSuiteEntry();
					entry.Result = (tr.Attributes["CLASS"] as string).convertTo<TestSuiteEntryResult>(TestSuiteEntryResult.NotSpecify);
					entry.SubmissionDate = tr.Columns[0].StringText.convertTo<DateTime>(DateTime.MinValue);
					entry.ID = tr.Columns[1].ChildAt(1).ToPlainTextString().toInt(0);
					entry.TestCaseID = tr.Columns[2].Children[0].ToPlainTextString().toInt(0);
					entry.Branch = tr.Columns[5].StringText;
					result.Add(entry);
				}
				catch (Exception) {
					Console.writeLine("Error process row {0} of page {1}", i, page);
				}
			}
		}

	}
}
